Implement GCID parsing in unicsv and extend ID to 64bit (#501)

* Implement GCID parsing in unicsv and extend ID to 64bit

This reimplements the parser and adds some test cases. The test
cases have been validated with several online converters, so this
hopefully works better than the previous implementation.

This also fixes an endless loop that afl managed to trigger.

* Fix clazy warning in unicsv

* Add GC-ID test cases, documentation, int parsing

- Explicitly do the integer parsing with base 10, otherwise hex
  values and octal values would be accepted with 0x and 0 prefix.
- Add testcases that cover GC-ID parsing, including some invalid cases
- Unify the terminology (GC-Code vs GC-ID)
- Add examples to the unicsv documentation showing that both GC-ID
  and GC-Code are accepted

diff --git a/defs.h b/defs.h
index b1a9fc486919f158ab6a672e5c7b7f069df7965a..6cd0bd00fb0a4e8b8f8da4d3bfa09dab9a35bec7 100644 (file)
--- a/defs.h
+++ b/defs.h
@@ -286,7 +286,7 @@ public:
     placer_id(0),
     favorite_points(0)
   {}
-  int id; /* The decimal cache number */
+  long long id; /* The decimal cache number */
   geocache_type type:5;
   geocache_container container:4;
   unsigned int diff:6; /* (multiplied by ten internally) */

diff --git a/gpx.cc b/gpx.cc
index 8c19589a33668b41fc03108f169cb059cb040a2f..cb1be39b66f6dbd890d13ce06e9e654b50668f3f 100644 (file)
--- a/gpx.cc
+++ b/gpx.cc
@@ -207,7 +207,7 @@ GpxFormat::tag_gs_cache(const QXmlStreamAttributes& attr)
   geocache_data* gc_data = wpt_tmp->AllocGCData();
 
   if (attr.hasAttribute("id")) {
-    gc_data->id = attr.value("id").toString().toInt();
+    gc_data->id = attr.value("id").toString().toLongLong();
   }
   if (attr.hasAttribute("available")) {
     if (attr.value("available").compare(QLatin1String("True"), Qt::CaseInsensitive) == 0) {

diff --git a/reference/gc/gcid-input.csv b/reference/gc/gcid-input.csv
new file mode 100644 (file)
index 0000000..d4d81a1
--- /dev/null
+++ b/reference/gc/gcid-input.csv
@@ -0,0 +1,57 @@
+lat,lon,gcid
+1.0, 2.0, GC1
+1.0, 2.0, GC01
+1.0, 2.0, GC001
+1.0, 2.0, GC0001
+1.0, 2.0, GC0010
+1.0, 2.0, GC0100
+1.0, 2.0, GC1000
+1.0, 2.0, GCS000
+1.0, 2.0, GCFFFF
+1.0, 2.0, GC0G000
+1.0, 2.0, GC10000
+1.0, 2.0, GCG0000
+1.0, 2.0, GCZ000O
+1.0, 2.0, GCZ000S
+1.0, 2.0, GCZZZZZ
+1.0, 2.0, GCZZZZZZ
+1.0, 2.0, GCZZZZZZZ
+1.0, 2.0, GCZZZZZZZZ
+1.0, 2.0, GCZZZZZZZZZ
+1.0, 2.0, GCZZZZZZZZZZ
+1.0, 2.0, GCZZZZZZZZZZZ
+1.0, 2.0, GCZZZZZZZZZZZZ
+1.0, 2.0, GC0
+1.0, 2.0, GC00
+1.0, 2.0, GC000
+1.0, 2.0, GC0000
+1.0, 2.0, GC0G00
+1.0, 2.0, GC0H00
+1.0, 2.0, GC0I00
+1.0, 2.0, GC0J00
+1.0, 2.0, GC0K00
+1.0, 2.0, GC0L00
+1.0, 2.0, GC0M00
+1.0, 2.0, GC0O00
+1.0, 2.0, GC0P00
+1.0, 2.0, GC0Q00
+1.0, 2.0, GC0R00
+1.0, 2.0, GC0T00
+1.0, 2.0, GC0U00
+1.0, 2.0, GC0V00
+1.0, 2.0, GC0W00
+1.0, 2.0, GC0X00
+1.0, 2.0, GC0Y00
+1.0, 2.0, GC0Z00
+1.0, 2.0, GCZ000I
+1.0, 2.0, GCZ000L
+1.0, 2.0, GCZ000U
+1.0, 2.0, GCZZZZZZZZZZZZZ
+1.0, 2.0, ABCDEF
+1.0, 2.0, 0x1234
+1.0, 2.0, 12F456
+1.0, 2.0, 12.456
+1.0, 2.0, 12 456
+1.0, 2.0, 012345
+1.0, 2.0, 123456
+1.0, 2.0, 787662783788138640

diff --git a/reference/gc/gcid-output.csv b/reference/gc/gcid-output.csv
new file mode 100644 (file)
index 0000000..0dc67c5
--- /dev/null
+++ b/reference/gc/gcid-output.csv
@@ -0,0 +1,57 @@
+No,Latitude,Longitude,Name,GCID\r
+1,1.000000,2.000000,"WPT001",1\r
+2,1.000000,2.000000,"WPT002",1\r
+3,1.000000,2.000000,"WPT003",1\r
+4,1.000000,2.000000,"WPT004",1\r
+5,1.000000,2.000000,"WPT005",16\r
+6,1.000000,2.000000,"WPT006",256\r
+7,1.000000,2.000000,"WPT007",4096\r
+8,1.000000,2.000000,"WPT008",20480\r
+9,1.000000,2.000000,"WPT009",65535\r
+10,1.000000,2.000000,"WPT010",65536\r
+11,1.000000,2.000000,"WPT011",512401\r
+12,1.000000,2.000000,"WPT012",14365216\r
+13,1.000000,2.000000,"WPT013",27294510\r
+14,1.000000,2.000000,"WPT014",27294515\r
+15,1.000000,2.000000,"WPT015",28218030\r
+16,1.000000,2.000000,"WPT016",887092560\r
+17,1.000000,2.000000,"WPT017",27512202990\r
+18,1.000000,2.000000,"WPT018",852890626320\r
+19,1.000000,2.000000,"WPT019",26439621749550\r
+20,1.000000,2.000000,"WPT020",819628286569680\r
+21,1.000000,2.000000,"WPT021",25408476895993710\r
+22,1.000000,2.000000,"WPT022",787662783788138640\r
+23,1.000000,2.000000,"WPT023",\r
+24,1.000000,2.000000,"WPT024",\r
+25,1.000000,2.000000,"WPT025",\r
+26,1.000000,2.000000,"WPT026",\r
+27,1.000000,2.000000,"WPT027",\r
+28,1.000000,2.000000,"WPT028",\r
+29,1.000000,2.000000,"WPT029",\r
+30,1.000000,2.000000,"WPT030",\r
+31,1.000000,2.000000,"WPT031",\r
+32,1.000000,2.000000,"WPT032",\r
+33,1.000000,2.000000,"WPT033",\r
+34,1.000000,2.000000,"WPT034",\r
+35,1.000000,2.000000,"WPT035",\r
+36,1.000000,2.000000,"WPT036",\r
+37,1.000000,2.000000,"WPT037",\r
+38,1.000000,2.000000,"WPT038",\r
+39,1.000000,2.000000,"WPT039",\r
+40,1.000000,2.000000,"WPT040",\r
+41,1.000000,2.000000,"WPT041",\r
+42,1.000000,2.000000,"WPT042",\r
+43,1.000000,2.000000,"WPT043",\r
+44,1.000000,2.000000,"WPT044",\r
+45,1.000000,2.000000,"WPT045",\r
+46,1.000000,2.000000,"WPT046",\r
+47,1.000000,2.000000,"WPT047",\r
+48,1.000000,2.000000,"WPT048",\r
+49,1.000000,2.000000,"WPT049",\r
+50,1.000000,2.000000,"WPT050",\r
+51,1.000000,2.000000,"WPT051",\r
+52,1.000000,2.000000,"WPT052",\r
+53,1.000000,2.000000,"WPT053",\r
+54,1.000000,2.000000,"WPT054",12345\r
+55,1.000000,2.000000,"WPT055",123456\r
+56,1.000000,2.000000,"WPT056",787662783788138640\r

diff --git a/testo.d/unicsv.test b/testo.d/unicsv.test
index 96140a4376fca002ece5fb5733d53ca4685b402e..03383562a83327dc94dbe05892b57989773c5c2a 100644 (file)
--- a/testo.d/unicsv.test
+++ b/testo.d/unicsv.test
@@ -15,6 +15,9 @@ gpsbabel -i gpx -f ${REFERENCE}/gc/GC7FA4.gpx -o unicsv,utc=0 -F ${TMPDIR}/gcuni
 gpsbabel -i unicsv,utc=0 -f ${REFERENCE}/gc/GC7FA4~unicsv.csv -o unicsv,utc=0 -F  ${TMPDIR}/gcunicsv-2.csv
 compare ${TMPDIR}/gcunicsv-1.csv ${TMPDIR}/gcunicsv-2.csv
 
+gpsbabel -i unicsv -f ${REFERENCE}/gc/gcid-input.csv -o unicsv -F ${TMPDIR}/gcid-output.csv
+compare ${REFERENCE}/gc/gcid-output.csv ${TMPDIR}/gcid-output.csv
+
 # check header detection features
 gpsbabel -i unicsv,utc=0 -f ${REFERENCE}/headerdetection.unicsv -x transform,trk=wpt -o gpx,garminextensions -F ${TMPDIR}/headerdetection~unicsv.gpx
 compare ${REFERENCE}/extensiondata~unicsv.gpx ${TMPDIR}/headerdetection~unicsv.gpx

diff --git a/unicsv.cc b/unicsv.cc
index cd0758499c3151f7cf2fbf5005666f439ddac7b6..50ed4e6bb3a4179cf31d1a18e2a3ed61ed07bd64 100644 (file)
--- a/unicsv.cc
+++ b/unicsv.cc
@@ -323,23 +323,61 @@ static QVector<arglist_t> unicsv_args = {
 
 /* helpers */
 
-// There is no test coverage of this and it's been wrong for years and
-// nobody has noticed...
-static int
-unicsv_parse_gc_id(const QString& str)
+// Parse GC-Code / geo cache reference code into int64 (GC-ID)
+// (see also https://api.groundspeak.com/documentation#referencecodes)
+static long long
+unicsv_parse_gc_code(const QString& str)
 {
-  int res = 0;
-  const QString kBase35 = "0123456789ABCDEFGHJKMNPQRTVWXYZ"; //  ILOSU are omitted.
-  if (str.startsWith("GC")) {
-    int base35 = str.size() > 6; // above GCFFFF?
-    QString s = str.mid(2);
-    while (!s.isEmpty()) {
-      res = res * 16 + kBase35.indexOf(s[0]);
-      s = str.mid(1);
+  if (! str.startsWith("GC")) {
+    return 0;
+  }
+
+  // Remove "GC" prefix
+  QString s = str.mid(2);
+  // Replacements according to groundspeak api documentation
+  s.replace('S', '5');
+  s.replace('O', '0');
+  // Remove leading zeros. some online converters do that as well.
+  while (s.startsWith('0')) {
+    s.remove(0, 1);
+  }
+
+  // We have these cases:
+  // *  1-3 digits                   => base 16
+  // *    4 digits, first one is 0-F => base 16
+  // *    4 digits, first one G-Z    => base 31
+  // * 5-12 digits                   => base 31
+  // *  13- digits                   => exceeds int64_t
+  //
+  int base;
+  const QString kBase31 = "0123456789ABCDEFGHJKMNPQRTVWXYZ"; //  ILOSU are omitted.
+  if (s.size() >= 1 && s.size() <= 3) {
+    base = 16;
+  } else if (s.size() == 4) {
+    if (kBase31.indexOf(s[0]) < 16) {
+      base = 16;
+    } else {
+      base = 31;
     }
-    if (base35) {
-      res -= 411120;
+  } else if (s.size() >= 5 && s.size() <= 12) {
+    base = 31;
+  } else {
+    return 0;
+  }
+
+  long long res = 0;
+  for (auto c : qAsConst(s)) {
+    int val = kBase31.indexOf(c);
+    if (val < 0 || (base == 16 && val > 15)) {
+      return 0;
     }
+    res = res * base + val;
+  }
+  if (base == 31) {
+    res -= 411120;
+  }
+  if (res < 0) {
+    res = 0;
   }
   return res;
 }
@@ -1010,9 +1048,13 @@ unicsv_parse_one_line(const QString& ibuf)
       switch (unicsv_fields_tab[column]) {
 
       case fld_gc_id:
-        gc_data->id = value.toInt();
-        if (gc_data->id == 0) {
-          gc_data->id = unicsv_parse_gc_id(value);
+        // First try to decode as numeric GC-ID (e.g. "575006").
+        // If that doesn't succedd, try to decode as GC-Code
+        // (e.g. "GC1234G").
+        bool ok;
+        gc_data->id = value.toLongLong(&ok, 10);
+        if (!ok) {
+          gc_data->id = unicsv_parse_gc_code(value);
         }
         break;
       case fld_gc_type:

diff --git a/xmldoc/formats/unicsv.xml b/xmldoc/formats/unicsv.xml
index ba2884778d6236765aea6b81d551ec7d39e5b273..333baf0de5cf9b7435733ba62b4ddde97cc884cf 100644 (file)
--- a/xmldoc/formats/unicsv.xml
+++ b/xmldoc/formats/unicsv.xml
@@ -32,7 +32,7 @@
       exported = Geocache export date
       found =    <link linkend="style_def_geofound">Geocache last found date</link>
       fix =      3d, 2d, etc.
-      gcid =     Geocache cache id
+      gcid =     Geocache cache id. This accepts GC-ID ("575006") and GC-Code ("GC1234G").
       geschw =   Geschwindigkeit (speed)
       hdop =     Horizontal dilution of precision
       head =     Heading / Course true